package pipeline;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.LinkedList;
import java.util.List;
import java.util.Map.Entry;

import setting.BioNLPSettings;
import util.io.FileUtil;
import util.ling.Porter;
import abs.syn.Dependency;
import abs.syn.DependencyPath;
import abs.syn.Parse;
import classifier.SVMWrapper;
import def.AbstractText;
import def.BioNLPDocument;
import def.Event;
import def.EventType;
import def.Protein;
import def.TSpan;
import exp.Count;

public class TriggerDetection {
	protected TriggerDetection(){
		
	}
	public LinkedList<String> dict = null;
	public/* static*/ HashSet<String> parentDictionary = new HashSet<String>(), bindingDictionary = new HashSet<String>();
	public /*static*/ Hashtable<String,Hashtable<String, Integer>> freq  = null;
	public static Hashtable<String, Integer> str2id = new Hashtable<String, Integer>();
	public static TriggerDetection getTD(){
		if (BioNLPSettings.getValue("TRIGGER_DETECTOR").equals(BioNLPSettings.TRIGGER_DICT_DETECTOR)){
			return new TriggerDetection();
		}
		else if(BioNLPSettings.getValue("TRIGGER_DETECTOR").equals( BioNLPSettings.TRIGGER_SVM_DETECTOR)){
			return new TriggerDetectionSVM();
		}
		else if(BioNLPSettings.getValue("TRIGGER_DETECTOR").equals( BioNLPSettings.TRIGGER_ORACLE_DETECTOR)){
			return new TriggerDetectionOracle();
		}
		else if (BioNLPSettings.getValue("TRIGGER_DETECTOR").equals( BioNLPSettings.TRIGGER_REFINER_DETECTOR)){
			return new TriggerDetectionRefiner();
		}
		else{
			return new TriggerDetection();
		}
	}

	public static void main(String[] args){
//				test();

		//		new TriggerDetection().learnDict();
//						System.exit(-1);

//		TriggerDetection td = new TriggerDetectionSVM();
		TriggerDetection td = new CotrainTrigger();
		td.init();
		td.str2id.clear();
		BioNLPSettings.init("train.conf");
		td.learn();
		
	}

	public void test(){
		// read a dictionary
//		BioNLPSettings.init("dev.conf");
//		TriggerDetection td = new TriggerDetection();
//		td.init();		
		//		triggerWordForEventType("data/BioNLP09/training/");

		String path = "data/BioNLP09/dev/";

		//		.parsePath = BioNLPSettings.devParsePath;
		Count sum = new Count();
		for (String filename :  new File(path).list()){
			//									String filename = "9159166.txt";
			if (filename.endsWith(".txt")){
				String filePrefix = filename.replace(".txt", "");
				System.out.println(filename+ " to be processed");
				sum.add(processDocument(path, filePrefix));
				System.out.println(sum);
			}
		}
	}

	public void init(){
		dict = readDictionary(BioNLPSettings.TRIGGER_DICT);
		// read parent and binding dicts too
		String[] lines = FileUtil.getTextFromFile(BioNLPSettings.BIND_TRIGGER_DICT).split("\n");
		for (String line: lines){
			bindingDictionary.add(line.replace("-", " "));
		}
		lines = FileUtil.getTextFromFile(BioNLPSettings.REG_TRIGGER_DICT).split("\n");
		for (String line: lines){
			parentDictionary.add(line.replace("-", " "));
		}

		FileInputStream fis = null;
		ObjectInputStream in = null;
		try
		{
			fis = new FileInputStream("trigger.freq.bin");
			in = new ObjectInputStream(fis);
			freq = (Hashtable<String,Hashtable<String, Integer>>)in.readObject();
			
			in.close();
			for (String key :freq.keySet()){
				for (String key2: freq.get(key).keySet())
					System.err.println("[FREQ]"+key+":"+key2+":"+freq.get(key).get(key2));
			}
		}
		catch(Exception ex)
		{
			ex.printStackTrace();
		}
	}

	private static LinkedList<String> readDictionary(String string) {
		String[] lines = FileUtil.getTextFromFile(string).split("\n");
		LinkedList<String> res =new LinkedList<String>();
		for (String line: lines){
			if (line.trim().length()==0 || line.startsWith("//"))
				continue;
			String[] items = line.split("\t");
			// filter out some common words, e.g. via ...
			if (items.length != 2)
				System.err.println(line+"   trigger wrong");
			if (items[0].length() <=BioNLPSettings.getIntValue("triggerLength")
					|| Integer.parseInt(items[1])<=BioNLPSettings.getIntValue("triggerFrequency"))
				continue;
			res.add(items[0].replace("-", " "));

			//			if(items.length > 2)
			//				System.err.println("sss");
		}
		Collections.sort(res, new Comparator<String>(){
			@Override
			public int compare(String arg0, String arg1) {
				return new Integer(arg0.length()).compareTo(arg1.length());
			}
		});
		return res;
	}

	public static Porter stemmer = new Porter();

	

	/**
	 * if the span is overlapping with protein names
	 */
	protected  void checkValidTrigger(BioNLPDocument doc, TSpan ts){
		// remove punc
		if (ts.text.matches("\\p{Punct}")){
			ts.startIdx = -1;
			return;
		}
		for (Protein p: doc.proteins){
			// if p.startIdx OR p.endIdx is with the span
			if ((p.tspan.startIdx>= ts.startIdx && p.tspan.startIdx < ts.endIdx)){
				//				System.err.println("[CHECKTRIGGER] OLD tspan = "+ts+" , protein = "+ p);
				if (p.tspan.endIdx < ts.endIdx){
					ts.startIdx = p.tspan.endIdx;
					if (doc.text.text.charAt(ts.startIdx)=='-')
						ts.startIdx += 1;
					if (ts.startIdx==ts.endIdx)
						ts.startIdx = -1;
				}
				else{
					if (p.tspan.startIdx != ts.startIdx){
						ts.endIdx = p.tspan.startIdx;
					}
					else{
						ts.startIdx = -1;
					}
				}
				//				System.err.println("[CHECKTRIGGER] NEW tspan = "+ts+" , protein = "+ p);
				break;
			}
			else if  (p.tspan.endIdx > ts.startIdx && p.tspan.endIdx <= ts.endIdx){
				//				System.err.println("[CHECKTRIGGER] OLD tspan = "+ts+" , protein = "+ p);
				if (p.tspan.startIdx >= ts.startIdx){
					ts.startIdx = p.tspan.endIdx;
					if (doc.text.text.charAt(ts.startIdx)=='-')
						ts.startIdx += 1;
					if (ts.startIdx==ts.endIdx)
						ts.startIdx = -1;
				}
				else{
					if (p.tspan.endIdx != ts.endIdx){
						ts.startIdx = p.tspan.endIdx;
						if (doc.text.text.charAt(ts.startIdx)=='-')
							ts.startIdx += 1;
					}
					else{
						ts.startIdx = -1;
					}
				}
				//				System.err.println("[CHECKTRIGGER] NEW tspan = "+ts+" , protein = "+ p);
				break;
			}
			else if (p.tspan.startIdx <= ts.startIdx && ts.endIdx <= p.tspan.endIdx){
				ts.startIdx = -1;
				break;
			}
		}
		
		
		
	}

	


	public ArrayList<TSpan> detectTriggers(BioNLPDocument doc){
		AbstractText text = (AbstractText)doc.text;
		String filePrefix = doc.fileId;

		ArrayList<TSpan> predictions  = new ArrayList<TSpan>();
		for (int i = 0; i < text.lines.length; i++){

			// de-hyphenate, lower-case
			String newLine = text.lines[i].replace("-", " ").toLowerCase();
			//			ArrayList<Integer> recordStart = new ArrayList<Integer>(), 
			//			recordEnd = new ArrayList<Integer>();
			//			ArrayList<TSpan> predictions = new ArrayList<TSpan>();
			// look up the dictionary
			for (int j = 0; j < dict.size(); j++){
				String word = dict.get(dict.size()-j-1);
				int start = 0, found = 0;
				// reverse order in trigger length
				while ((found = newLine.indexOf(word,start))!=-1){
					start = found+1;

					// should be a complete word
					if ((found!=0?newLine.substring(found-1, found).matches("\\w"):false)|| 
							(found+word.length()!= newLine.length()?newLine.substring(found+word.length(), found+word.length()+1).matches("\\w"):false) ){
						System.err.println("[TRIGGER_DETECT] dropping "+newLine.substring(found, found+word.length())+" @ "+ (found+ text.sentStartPos[i])+ ": "+word.length() );
						continue;
					}



					found += text.sentStartPos[i];
					boolean legal = true;
					// check if the span has already been labeled by another detected span
					for (int k = 0; k < predictions.size(); k++){
						if (found >= predictions.get(k).startIdx && found <= predictions.get(k).endIdx){
							legal = false;
							break;
						}
					}
					// check if the span has already been  labeled as a protein
					for (Protein p: doc.proteins){
						if (found >= p.tspan.startIdx && found <=  p.tspan.endIdx){
							legal = false;
							break;
						}
					}
					if (legal){
						// record the span
						TSpan newSpan = new TSpan();
						newSpan.startIdx = found;
						//						recordStart.add(found);
						newSpan.endIdx = found+word.length();
						newSpan.fileId = filePrefix;
						newSpan.text = word;
						predictions.add(newSpan);
						//						recordEnd.add(found+word.length());
					}
				}
			}
		}
		return predictions;
	}


	public Count processDocument(String path, String filePrefix){
		Count c = new Count();
		BioNLPDocument doc = new BioNLPDocument(path, filePrefix);
		Hashtable<String, Event> events = doc.events;
		AbstractText  text = (AbstractText)doc.text;
		if (!doc.valid){
			return c;
		}
		// detect triggers for each line
		Hashtable<TSpan, Boolean> detected = new Hashtable<TSpan, Boolean>();
		for (Event e: events.values()){
			//			c.all ++;
			detected.put(e.tspan, false);
		}

		ArrayList<TSpan> predictions = detectTriggers(doc);
		for (int i = 0; i < predictions.size(); i++){
			// check the correctness
			boolean miss = true;
			TSpan ts = predictions.get(i);
			for (Event event: events.values()){
				//							if (arg instanceof Event){
				//								Event event = ((Event)arg);
				if (BioNLPSettings.getValue("evalMethod") == BioNLPSettings.EVAL_APPROX){
					String[] pair = event.tspan.getEvalString(doc).split("-");
					if (Integer.parseInt(pair[0]) <= ts.startIdx
							&& Integer.parseInt(pair[1]) >=  ts.endIdx){
						if (miss&&!detected.get(event.tspan))
							c.num++;
						miss = false;
						detected.put(event.tspan, true);
						//									break;
					}
				}
				else if (BioNLPSettings.getValue("evalMethod") == BioNLPSettings.EVAL_STRICT){
					if (event.tspan.startIdx == ts.startIdx
							&& event.tspan.endIdx ==  ts.endIdx){
						if (miss&&!detected.get(event.tspan))
							c.num++;
						miss = false;
						detected.put(event.tspan, true);
						//									break;
					}
				}
				//							}
			}
			if (miss){
				c.fp++;
			}
		}
		for (int i = 0; i < text.lines.length; i++){


			System.out.println("["+i+"]=="+text.lines[i]+"==");

			System.out.println("Predictions: ");
			for (int k = 0 ; k < predictions.size(); k++){
				if (text.getSentenceIdByPos(predictions.get(k).startIdx) == i){
					System.out.println(predictions.get(k).text+" ("+ (text.sentStartPos[i]+predictions.get(k).startIdx)+", "+( text.sentStartPos[i]+predictions.get(k).endIdx)+")");
				}
			}

			System.out.println("Ground Truth: ");
			for (Event event: events.values()){
				if (text.getSentenceIdByPos(event.tspan.startIdx)==i){
					System.out.println(event.eid+"   "+event.tspan+"   "+text.getSentenceIdByPos(event.tspan.startIdx)+"   "+i);
				}
			}
		}
		System.out.println("===NOT FOUND===");
		HashSet<String> visitedTspans = new HashSet<String>();
		for (Event e: events.values()){
			if (!visitedTspans.contains(e.tid)){
				c.all ++;
				visitedTspans.add(e.tid);
			}

			if (!detected.get(e.tspan))
				System.out.println(e.tspan);
		}
		//		
		//		for (Event e: freq.keySet()){
		//			c.all++;
		//			c.num += freq.get(e);
		//		}
		return c;
	}

	/**
	 * build the dictionary(-ies)
	 * @param dataPath
	 */
	public void learn(){
		BioNLPSettings.init("train.conf");
		String dataPath = BioNLPSettings.getValue("dataPath");
		String[] files = new File(dataPath).list();
		Porter stemmer = new Porter();
		Hashtable<String, String> multiwords = new Hashtable<String, String>();
		// (event_type, [string, freq])
		freq = new Hashtable<String,Hashtable<String, Integer>>();
		for (String file: files){
			if (file.endsWith(".a2")){
				Hashtable<String, String> id2str = new Hashtable<String,String>();
				String[] lines = FileUtil.getTextFromFile(dataPath+file).split("\n");
				for (String line: lines){
					if (line.startsWith("T")){
						String[] items = line.split("\t");
						String str = items[2].toLowerCase();
						items = items[1].split(" ");
						String type = items[0];
						if (type.equals("Entity")){
							continue;
						}
						if (!str.contains(" ")){
							str = /*str+":"+*/stemmer.stripAffixes(str);
						}
						else{
							multiwords.put(str, items[0]);
							//							continue;
						}
						//						id2str.put(items[0], items[2]);


						if (!freq.containsKey(type))
							freq.put(type, new Hashtable<String, Integer>());
						if (!freq.get(type).containsKey(str))
							freq.get(type).put(str,0);
						freq.get(type).put(str, freq.get(type).get(str)+1);
					}
					/*else if (line.startsWith("E")){
						String[] items = line.split("\\s+");
						String[] pair = items[1].split(":");
						if (!freq.containsKey(pair[0]))
							freq.put(pair[0], new Hashtable<String, Integer>());
						String str = id2str.get(pair[1]);
						if (str == null)
							System.err.println(id2str + "  "+ pair[1]);
						//						if (str.contains(" "))
						//							str = "MULTIWORDS";
						//						else
						//							{stemmer.add(str.toCharArray(), str.length());
						//								stemmer.stem();
						//								str = stemmer.toString();
						//							}
						if (!freq.get(pair[0]).containsKey(str))
							freq.get(pair[0]).put(str,0);
						freq.get(pair[0]).put(str, freq.get(pair[0]).get(str)+1);
					}*/
				}
			}
		}

		for (String key: multiwords.keySet()){
			String[] items = key.split("(\\s+|\\-)");
			System.err.println("[MULTIWORDS] "+key+"  :  "+multiwords.get(key));
			for (String item:items){
				System.err.print("[MULTIWORDS]   word: "+ item );
				for (String type: freq.keySet()){
					if (freq.get(type).containsKey(stemmer.stripAffixes(item.toLowerCase()))){
						System.err.print(" *"+type+"*");
					}
				}
				System.err.println();
			}
		}

		for (String type: freq.keySet()){
			System.out.println("TYPE: "+type +"<br />");

			int sum = 0;
			StringBuilder sb = new StringBuilder();
			sb.append("<table border=1><tr><td>trigger</td><td>#occurrence</td></tr>");
			LinkedList<String> list = new LinkedList<String>(freq.get(type).keySet());
			Collections.sort(list);
			for (int i = 0; i < list.size(); i++){
				String key = list.get(i);
				sb.append("<tr><td>");
				sb.append(key+"</td><td>"+freq.get(type).get(key));
				sb.append("</td></tr>");
				sum += freq.get(type).get(key);
			}
			sb.append("</table>");
			//			for (Entry<String, Integer>entry: freq.get(type).entrySet()){
			//				System.out.println("="+entry.getKey()+" "+entry.getValue());
			//				sum+=entry.getValue();
			//			}
			System.out.println(sum+" occurrences <br />");
			System.out.println(freq.get(type).size()+" entries <br />");
			System.out.println(sb.toString());



		}

		if (true){
			return;
		}
		StringBuilder dict = new StringBuilder();
		HashSet<String> parentDictionary = new HashSet<String>(), bindingDictionary = new HashSet<String>();

		FileOutputStream fos = null;
		ObjectOutputStream out = null;
		try
		{
			fos = new FileOutputStream("trigger.freq.bin");
			out = new ObjectOutputStream(fos);
			out.writeObject(freq);
			out.close();
		}
		catch(IOException ex)
		{
			ex.printStackTrace();
		}

		for (String type: freq.keySet()){
			for (String key: freq.get(type).keySet()){
				dict.append(key.toLowerCase() +"\t"+freq.get(type).get(key)+"\n");
			}

			if (type.endsWith("egulation")){
				// regulation
				for (String key: freq.get(type).keySet()){
					//					dict.append(key.toLowerCase() +"\t"+freq.get(type).get(key)+"\n");
					parentDictionary.add(key.toLowerCase());
				}

			}
			else if (type.equals("Binding")){
				// binding
				for (String key: freq.get(type).keySet()){
					bindingDictionary.add(key.toLowerCase());
				}
			}
		}
		FileUtil.writeTextToFile(dict.toString(), "trigger.dict");
		StringBuilder sb = new StringBuilder();
		for (String key: parentDictionary){
			sb.append(key+"\n");
		}
		FileUtil.writeTextToFile(sb.toString(), "parentTrigger.dict");
		sb = new StringBuilder();
		for (String key: bindingDictionary){
			sb.append(key+"\n");
		}
		FileUtil.writeTextToFile(sb.toString(), "bindingTrigger.dict");
	}


	

}
